#! /usr/bin/env python
# -*- coding:utf-8 -*-


'''
Author:非佛语不语
--
Date:2018-XX-XX
'''

'''
爬取以下站点中各个明星图片
分别单独建文件夹存放。
起始URL地址：http://www.mm131.com/mingxing
'''

import os, requests, re, logging
from bs4 import BeautifulSoup

# 设置日志等级为:INFO
logging.basicConfig(level=logging.INFO)


# 自定义store_stars_img()获取明星的每一张图片
def store_stars_img(stars_url, store_stars_dir):
    stars_text = requests.get(stars_url).text
    soup = BeautifulSoup(stars_text, 'lxml')
    stars_src = soup.select('.content-pic img')[0]['src']
    filename = stars_src.split('/')[-1]
    filename = os.path.join(store_stars_dir, filename)
    with open(filename, 'wb')as f:
        # 加上页面的请求头
        headers = {
            'Referer': stars_url,
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
        }
        # 获取stars_src的二进制内容:gstars_content
        stars_content = requests.get(stars_src, headers=headers).content
        f.write(stars_content)


# 自定义方法store_page_stars()处理一个明星的多张图片
def store_page_stars(middle_href, store_stars_dir):
    temp_href = middle_href.split('.html')[0]
    # print(temp_href)
    stars_text = requests.get(middle_href).text
    soup = BeautifulSoup(stars_text, 'lxml')
    max_page_num = int(soup.select('.content-page .page-en')[-1].get_text())
    stars_url_list = []
    for page_num in range(1, max_page_num + 1):
        if page_num == 1:
            temp_url = middle_href
            stars_url_list.append(temp_url)
        else:
            stars_url_list.append(f'{temp_href}_{page_num}.html')
    for stars_url in stars_url_list:
        store_stars_img(stars_url, store_stars_dir)


# 自定义home_page()方法
def home_page(page_url):
    # 创建文件夹stars作为图片根目录
    store_dir = 'stars'
    os.makedirs(store_dir, exist_ok=True)
    # 请求明星的每个主页面page_url内容
    r = requests.get(page_url)
    r.encoding = 'gbk'
    home_text = r.text
    # 构造soup对象解析该页面
    soup = BeautifulSoup(home_text, 'lxml')
    middle_ahref_list = soup.find('dl', class_='list-left').find_all('dd')[0:20]

    for middle_ahref in middle_ahref_list:
        # 以页面中明星标题为名创建目录
        starname = middle_ahref.get_text()
        store_stars_dir = os.path.join(store_dir, starname)
        os.makedirs(store_stars_dir, exist_ok=True)
        # 使用正则获取middle_ahref标签中的herf熟悉
        middle_ahref = str(middle_ahref)
        ahref = r'href="(.*?)"'
        middle_href = str(re.findall(ahref, middle_ahref))
        middle_href = re.sub('[\[\]\',]', '', middle_href)

        # 开始下载页面图片提示信息
        logging.info(f'正在下载--{starname}--页面里面的所有图片...')
        store_page_stars(middle_href, store_stars_dir)


# 自定义main()方法
def main():
    # 原始链接
    original_url = 'http://www.mm131.com/mingxing'
    # 请求原始链接original_url
    main_text = requests.get(original_url).text
    # 构造soup对象解析home_text
    soup = BeautifulSoup(main_text, 'lxml')
    outer_ahref_list = soup.find_all(class_=['page_now', 'page-en'])[1:7]
    page_url_list = ['http://www.mm131.com/mingxing']
    for outer_ahref in outer_ahref_list:
        # print(outer_ahref)    #测试最外层页面a标签:<span class="page_now">1</span>
        # <a class="page-en" href="list_5_2.html">2</a>
        outer_href = outer_ahref['href']
        every_page_url = f'{original_url}/{outer_href}'
        page_url_list.append(every_page_url)
    for page_url in page_url_list:
        home_page(page_url)


# 程序主入口
if __name__ == '__main__':
    main()